﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Collections;
using System.Threading;
using System.IO;
using System.Text.RegularExpressions;
using System.Diagnostics;
using System.Globalization;


namespace TaskCrawl
{
    public partial class Form1 : Form
    {
        public static Queue<String> q = new Queue<string>();
        public static Stopwatch watch = new Stopwatch();
        public static int pagenum;
        public static object locker1 = new object();
        public static object locker2 = new object();


        public Form1()
        {
            InitializeComponent();
            this.listView1.Columns.Add("webpages");
            this.listView1.Columns.Add("count words");
            this.listView1.View = System.Windows.Forms.View.Details;
            listView1.Columns[0].Width = 450;
            listView1.Columns[1].Width = 100;
        }

        public string getMainContent()
        {
            string urlToCrawl = textBox1.Text;
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlToCrawl);
            req.Method = "GET";
            HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
            string htmlCharset = "utf-8";
            Encoding htmlEncoding = Encoding.GetEncoding(htmlCharset);
            StreamReader sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);
            string mainHtml = sr.ReadToEnd();//Substring(0,10000);
            return mainHtml;
        }

        public delegate void ProcessDelegate();

        public int wordCount(string rrh)
        {
            string content = rrh;
            string pattern = @"[\u4E00-\u9fA5]";
            int totalnum = Regex.Matches(content, pattern).Count;
            return totalnum;
        }

        public void Download()
        {
            int countword;
            while (true)
            {
                string url;
                while (q.Count > 0)
                {
                    lock (locker1)
                    {
                        url = q.Dequeue();
                    }
                    try
                    {
                        HttpWebRequest rr = (HttpWebRequest)WebRequest.Create(url);
                        rr.Method = "GET";
                        HttpWebResponse resp = (HttpWebResponse)rr.GetResponse();

                        string htmlCharset = "utf-8";
                        Encoding htmlEncoding = Encoding.GetEncoding(htmlCharset);
                        StreamReader sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);
                        string rh = sr.ReadToEnd();

                        countword = wordCount(rh);

                        lock (locker2)
                        {
                            ProcessDelegate showProcess = delegate ()
                            {
                                ListViewItem it = new ListViewItem();
                                it.Text = url;
                                it.SubItems.Add(countword.ToString());
                                listView1.Items.Add(it);
                                pagenum++;
                                label_num.Text = pagenum.ToString();
                            };
                            Invoke(showProcess);
                        }
                    }
                    catch (Exception e2)
                    {
                        MessageBox.Show(url + " download fail");
                    }
                }
            }
        }
        private void Click_btnCrawl(object sender, EventArgs e)
        {
            string mainweb = getMainContent();
            string links = @"(http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?)";
            MatchCollection getlinks = (new Regex(links)).Matches(mainweb);
            foreach (Match m in getlinks)
            {
                string url = (string)m.Value;
                q.Enqueue(url);
            }
            int total = q.Count;
            int dec = total / 10;

            Thread[] downloadThread;
            downloadThread = new Thread[q.Count];
        
            watch.Start();
            try
            {
                for (int i = 0; i < dec; i++)
                {
                    for (int j = 0; j < 10; j++)
                    {
                        while (total != 0)
                        {
                            ThreadStart startDownload = new ThreadStart(Download);
                            downloadThread[i + j * 10] = new Thread(startDownload);
                            downloadThread[i + j * 10].Start();
                            total--;
                          
                        }
                    }
                }
            }
            catch (ThreadAbortException e1)
            {
                MessageBox.Show(e1.Message);
            }
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }
    }
}
